package com.wanglei.rdd.dependency

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Spark02_rdddep {

  def main(args: Array[String]): Unit = {
    //
    val sparkConf = new SparkConf().setMaster("local[2]").setAppName("wc")
    val sc = new SparkContext(sparkConf)

    //
    val lines: RDD[String] = sc.textFile("datas/1.txt")
    println(lines.dependencies)
    println("*" * 50)
    val words: RDD[String] = lines.flatMap(_.split(" "))
    println(words.dependencies)
    println("*" * 50)
    val wordToOne: RDD[(String, Int)] = words.map(word => (word, 1))
    println(wordToOne.dependencies)
    println("*" * 50)
    val res: RDD[(String, Int)] = wordToOne.reduceByKey(_ + _)
    println(res.dependencies)
    println("*" * 50)

    res.collect().foreach(println(_))

    //
    sc.stop()
  }

}
